# 解析器
import re
import urlparse

from bs4 import BeautifulSoup


class HtmlParser(object):

    def parse(self, new_url, html_content):
        if new_url is None or html_content is None:
            return
        # BeautifulSoup中参数：参数1-网页，参数2-解析器，参数3-参数编码
        soup = BeautifulSoup(html_content, 'html.parser', from_encoding='utf-8')
        new_urls = self._get_new_urls(new_url, soup)
        new_data = self._get_new_data(new_url, soup)
        
        return new_urls, new_data

    # urljoin 函数可以将new_url 按new_url的格式拼接一个全新的url
    def _get_new_urls(self, new_url, soup):
        # 解析出来的url存放到set中
        new_urls = set()
        links = soup.find_all('a', href=re.compile(r"/item/"))
        for link in links:
            url = link['href']
            new_full_url = urlparse.urljoin(new_url, url)
            new_urls.add(new_full_url)
        return new_urls

    # 获取数据
    def _get_new_data(self, new_url, soup):
        result_data = {}

        # <dd class="lemmaWgt-lemmaTitle-title">
        # <h1>Python</h1>

        title_node = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1')
        result_data['title'] = title_node.get_text()

        # lemma-summary
        summary_node = soup.find('div', class_='lemma-summary')
        result_data['summary'] = summary_node.get_text()

        result_data['url'] = new_url

        return result_data

